CNN for image Classification

importing libraries

# importing libraries for CNN

import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Activation, Dense, Flatten, BatchNormalization, Conv2D, MaxPool2D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import categorical_crossentropy
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import confusion_matrix
import itertools
import os
import shutil
import random
import glob
import matplotlib.pyplot as plt
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
%matplotlib inline

# checking for GPU

physical_devices = tf.config.experimental.list_physical_devices('GPU')
print("Num GPUs Available: ", len(physical_devices))
#tf.config.experimental.set_memory_growth(physical_devices[0], True)

Num GPUs Available:  0

Creating and organizing the images into train, validation and test

• download only the train data from kaggle
• https://www.kaggle.com/c/dogs-vs-cats/data
• make a new directory called dogs-vs-cats and extract all the images in this directory
• change your directory to the dogs-vs-cats as shown below

# creating and organizing the data

dir = r"C:\Users\tejas\Desktop\work\Portfolio_projects\tensorflow\data\dogs-vs-cats"

os.chdir(dir)

# creating directories for dogs and cat images
if os.path.isdir('train\dog') is False:
    os.makedirs('train\dog')
    os.makedirs('train\cat')
    os.makedirs('valid\dog')
    os.makedirs('valid\cat')
    os.makedirs('test\dog')
    os.makedirs('test\cat')
    
    # iterating through the data to move the cat and dog images to respective directories
    # since we have 25000 images we just take a sample of them here
    for c in random.sample(glob.glob('cat*'), 500):
        shutil.move(c, 'train\cat')
    for c in random.sample(glob.glob('dog*'), 500):
        shutil.move(c, 'train\dog')
    for c in random.sample(glob.glob('cat*'), 100):
        shutil.move(c, 'valid\cat')
    for c in random.sample(glob.glob('dog*'), 100):
        shutil.move(c, 'valid\dog')
    for c in random.sample(glob.glob('cat*'), 50):
        shutil.move(c, 'test\cat')
    for c in random.sample(glob.glob('dog*'), 50):
        shutil.move(c, 'test\dog')
    

# Assigning train, valid and test respectively
# since we are already in the current directory we don't have to givethe complete path

train_path = 'train'
valid_path = 'valid'
test_path = 'test'

# using ImageDataGenerator and flow from directory
# By using ImageDataGenerator we can apply any random transformations on each training image as it is passed to the model one by one.
# The flow_from_directory() method takes a path of a directory and generates batches of augmented data. 
# The directory structure is very important when you are using flow_from_directory() method. 
# The flow_from_directory() assumes: The root directory contains at least two folders one for train and one for the test.

# preprocessing_function is leveraging pre trained weights
# target_size resizes all the images to the specified size
train_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.vgg16.preprocess_input) \
    .flow_from_directory(directory=train_path, target_size=(224,224), classes=['cat', 'dog'], batch_size=10)

valid_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.vgg16.preprocess_input) \
    .flow_from_directory(directory=valid_path, target_size=(224,224), classes=['cat', 'dog'], batch_size=10)

# Here shuffle=False because, later we have to compare the prediction of the image classes to their actual classes
test_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.vgg16.preprocess_input) \
    .flow_from_directory(directory=test_path, target_size=(224,224), classes=['cat', 'dog'], batch_size=10, shuffle=False)

Found 1000 images belonging to 2 classes.
Found 200 images belonging to 2 classes.
Found 100 images belonging to 2 classes.

# the above code creates a generator function and 
train_batches

<keras.preprocessing.image.DirectoryIterator at 0x1fe88e5df40>

# calling the iterator function next to pass the data one after another
# since our batch size is 10, 10 number of data is passed one after another as shown below

imgs, labels = next(train_batches)

imgs.shape

(10, 224, 224, 3)

labels.shape

(10, 2)

Plotting the train sample images

# creating a function to plot the images
# since batch size is 10 plotting 10 images

def plotImages(images_arr):
    fig, axes = plt.subplots(1, 10, figsize=(20,20))
    axes = axes.flatten()
    for img, ax in zip( images_arr, axes):
        ax.imshow(img)
        ax.axis('off')
    plt.tight_layout()
    plt.show()

# calling the function
# it can be observed that the image color is distorted this is because of the preprocessed weights of vgg16
# here [0. 1.] represents a dog and [1. 0.] represents a cat

plotImages(imgs)
print(labels)

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

[[0. 1.]
 [0. 1.]
 [1. 0.]
 [1. 0.]
 [0. 1.]
 [0. 1.]
 [1. 0.]
 [0. 1.]
 [1. 0.]
 [1. 0.]]

Building and Training a CNN model

# building a CNN sequential model
# padding=same does not reduce the image dimension after convolution

model = Sequential([
    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(224,224,3)),
    MaxPool2D(pool_size=(2, 2), strides=2),
    Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding = 'same'),
    MaxPool2D(pool_size=(2, 2), strides=2),
    Flatten(),
    Dense(units=2, activation='softmax')
])

model.summary()

Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_4 (Conv2D)           (None, 224, 224, 32)      896       
                                                                 
 max_pooling2d_4 (MaxPooling  (None, 112, 112, 32)     0         
 2D)                                                             
                                                                 
 conv2d_5 (Conv2D)           (None, 112, 112, 64)      18496     
                                                                 
 max_pooling2d_5 (MaxPooling  (None, 56, 56, 64)       0         
 2D)                                                             
                                                                 
 flatten_2 (Flatten)         (None, 200704)            0         
                                                                 
 dense_2 (Dense)             (None, 2)                 401410    
                                                                 
=================================================================
Total params: 420,802
Trainable params: 420,802
Non-trainable params: 0
_________________________________________________________________

# compiling the model for backpropogation
# categorical_crossentropy is used when there are more that 2 classes to predict, however it can be used for 2 classes also

model.compile(optimizer= Adam(learning_rate=0.0001), loss = 'categorical_crossentropy', metrics = ['accuracy'])

# fitting the model with validation data
# The train_batches has both the images and labels as a generator, hence we just need to pass x= train_batches 

model.fit(x= train_batches, validation_data= valid_batches, epochs = 10, verbose = 2)

Epoch 1/10
100/100 - 71s - loss: 15.9112 - accuracy: 0.5900 - val_loss: 4.1577 - val_accuracy: 0.5850 - 71s/epoch - 715ms/step
Epoch 2/10
100/100 - 55s - loss: 2.0310 - accuracy: 0.7580 - val_loss: 2.5093 - val_accuracy: 0.6150 - 55s/epoch - 553ms/step
Epoch 3/10
100/100 - 59s - loss: 0.5761 - accuracy: 0.8800 - val_loss: 2.6220 - val_accuracy: 0.6350 - 59s/epoch - 589ms/step
Epoch 4/10
100/100 - 65s - loss: 0.1804 - accuracy: 0.9490 - val_loss: 2.7146 - val_accuracy: 0.6400 - 65s/epoch - 645ms/step
Epoch 5/10
100/100 - 68s - loss: 0.0503 - accuracy: 0.9830 - val_loss: 3.1372 - val_accuracy: 0.6500 - 68s/epoch - 677ms/step
Epoch 6/10
100/100 - 65s - loss: 0.0446 - accuracy: 0.9830 - val_loss: 2.2216 - val_accuracy: 0.6550 - 65s/epoch - 654ms/step
Epoch 7/10
100/100 - 66s - loss: 0.0059 - accuracy: 0.9990 - val_loss: 2.0615 - val_accuracy: 0.6900 - 66s/epoch - 655ms/step
Epoch 8/10
100/100 - 66s - loss: 6.9752e-04 - accuracy: 1.0000 - val_loss: 2.1359 - val_accuracy: 0.6900 - 66s/epoch - 659ms/step
Epoch 9/10
100/100 - 61s - loss: 4.9878e-04 - accuracy: 1.0000 - val_loss: 2.1295 - val_accuracy: 0.6950 - 61s/epoch - 609ms/step
Epoch 10/10
100/100 - 58s - loss: 3.8161e-04 - accuracy: 1.0000 - val_loss: 2.1142 - val_accuracy: 0.7050 - 58s/epoch - 577ms/step

<keras.callbacks.History at 0x1fe89c55670>

Prediction / Inference

# plotting the test images
# calling the function which we created before to plot the images

test_imgs, test_labels = next(test_batches)
plotImages(test_imgs)
print(test_labels)

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

[[1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]
 [1. 0.]]

# These are the classes of the test batches
# 0 is for cat and 1 is for dog
# since we haven't shuffled the test data we have all cats on the top

test_batches.classes

array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])

# predicting the output from the test batch data

predictions = model.predict(x= test_batches, verbose=0)

# [1., 0.] represents that index 0 has 1 that has the highest probability hence the class is cat, since cat has the label 0
# [0., 1.] this says dog has the highest probability, since its label is 1

np.round(predictions)

array([[1., 0.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [1., 0.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [1., 0.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [1., 0.]], dtype=float32)

# creating a confusion matrix
# test_batches.classes to get the class labels
# np.argmax(predictions, axis= -1) to get the index of the most probable class

cm = confusion_matrix(y_true = test_batches.classes, y_pred= np.argmax(predictions, axis= -1))

# to know which index is cat and dog

test_batches.class_indices

{'cat': 0, 'dog': 1}

# code to plot the confusion matrix

from sklearn.metrics import ConfusionMatrixDisplay

disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['cat', 'dog'])

disp.plot(cmap=plt.cm.Blues)
plt.show()

We can see that the model is completely overfitting on training data hence the training accuracy is high and the testing accuracy is very low

Building Fine Tuned model using VGG16, a preprocessing technique

• Every image has RGB color scheme
• VGG16 subtracted the mean pixel of red with the red colors in the image, similarly with green and blue.
• This is the reason the image looks distorted when the preprocessing_function=tf.keras.applications.vgg16.preprocess_input is called
• Since the training data was preprocessed while training, the new data for testing also should be preprocessed and this can be acheived easily through VGG16

# download the model from keras applications

vgg16_model = tf.keras.applications.vgg16.VGG16()

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5
553467904/553467096 [==============================] - 253s 0us/step
553476096/553467096 [==============================] - 253s 0us/step

# summary of the VGG16 model
# note that VGG16 has 1000 prebuilt prediction classes
# for our use case we just have 2 classes to predict hence we would be changing that

vgg16_model.summary()

Model: "vgg16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 block1_conv1 (Conv2D)       (None, 224, 224, 64)      1792      
                                                                 
 block1_conv2 (Conv2D)       (None, 224, 224, 64)      36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, 112, 112, 64)      0         
                                                                 
 block2_conv1 (Conv2D)       (None, 112, 112, 128)     73856     
                                                                 
 block2_conv2 (Conv2D)       (None, 112, 112, 128)     147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, 56, 56, 128)       0         
                                                                 
 block3_conv1 (Conv2D)       (None, 56, 56, 256)       295168    
                                                                 
 block3_conv2 (Conv2D)       (None, 56, 56, 256)       590080    
                                                                 
 block3_conv3 (Conv2D)       (None, 56, 56, 256)       590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, 28, 28, 256)       0         
                                                                 
 block4_conv1 (Conv2D)       (None, 28, 28, 512)       1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, 28, 28, 512)       2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, 28, 28, 512)       2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, 14, 14, 512)       0         
                                                                 
 block5_conv1 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, 7, 7, 512)         0         
                                                                 
 flatten (Flatten)           (None, 25088)             0         
                                                                 
 fc1 (Dense)                 (None, 4096)              102764544 
                                                                 
 fc2 (Dense)                 (None, 4096)              16781312  
                                                                 
 predictions (Dense)         (None, 1000)              4097000   
                                                                 
=================================================================
Total params: 138,357,544
Trainable params: 138,357,544
Non-trainable params: 0
_________________________________________________________________

# The imported Model's type is Functional
# since we are working with sequential model right now, we need to convert it to sequential

type(vgg16_model)

keras.engine.functional.Functional

# initializing Sequential class to the object model
model = Sequential()

# iterating through all the layers except the last layer (because we need to change it to 2 class prediction layer)
# every layer is added into the Sequential model
# it can be seen that all the layers are added except the last layer
for layer in vgg16_model.layers[:-1]:
    model.add(layer)
    print(layer)

<keras.engine.input_layer.InputLayer object at 0x000001FE8FC04A90>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FCA2160>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC20370>
<keras.layers.pooling.MaxPooling2D object at 0x000001FE8FC04B80>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC81B50>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC04EB0>
<keras.layers.pooling.MaxPooling2D object at 0x000001FE8FB32040>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC8E340>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC8EBB0>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC867C0>
<keras.layers.pooling.MaxPooling2D object at 0x000001FE8FC8EB20>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC9BB50>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC72F40>
<keras.layers.convolutional.Conv2D object at 0x000001FE8C4C6DC0>
<keras.layers.pooling.MaxPooling2D object at 0x000001FE8FB32A90>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FB32730>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC72940>
<keras.layers.convolutional.Conv2D object at 0x000001FE8FC81F40>
<keras.layers.pooling.MaxPooling2D object at 0x000001FE8FB32F10>
<keras.layers.core.flatten.Flatten object at 0x000001FE8FC04A30>
<keras.layers.core.dense.Dense object at 0x000001FE8FCA2280>
<keras.layers.core.dense.Dense object at 0x000001FE8FC1BE80>

# They are exactly the same with the inbuild vgg16 model except this model does not have the last layer
model.summary()

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 block1_conv1 (Conv2D)       (None, 224, 224, 64)      1792      
                                                                 
 block1_conv2 (Conv2D)       (None, 224, 224, 64)      36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, 112, 112, 64)      0         
                                                                 
 block2_conv1 (Conv2D)       (None, 112, 112, 128)     73856     
                                                                 
 block2_conv2 (Conv2D)       (None, 112, 112, 128)     147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, 56, 56, 128)       0         
                                                                 
 block3_conv1 (Conv2D)       (None, 56, 56, 256)       295168    
                                                                 
 block3_conv2 (Conv2D)       (None, 56, 56, 256)       590080    
                                                                 
 block3_conv3 (Conv2D)       (None, 56, 56, 256)       590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, 28, 28, 256)       0         
                                                                 
 block4_conv1 (Conv2D)       (None, 28, 28, 512)       1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, 28, 28, 512)       2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, 28, 28, 512)       2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, 14, 14, 512)       0         
                                                                 
 block5_conv1 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, 7, 7, 512)         0         
                                                                 
 flatten (Flatten)           (None, 25088)             0         
                                                                 
 fc1 (Dense)                 (None, 4096)              102764544 
                                                                 
 fc2 (Dense)                 (None, 4096)              16781312  
                                                                 
=================================================================
Total params: 134,260,544
Trainable params: 134,260,544
Non-trainable params: 0
_________________________________________________________________

# now we need to set layer.trainable=false because VGG16 has already been trained on the features of cats and dogs
# We don't have to retrain the model again
# We are freezing the weights here

for layer in model.layers:
    layer.trainable = False

# adding the output layer

model.add(Dense(units=2, activation= 'softmax'))

# we can observe that the last output layer is 2
# Also the total trainable parameters are only 8194 because we freezed the trainable parameters before
# All the 8194 trainable parameters are only from the last output layer we added

model.summary()

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 block1_conv1 (Conv2D)       (None, 224, 224, 64)      1792      
                                                                 
 block1_conv2 (Conv2D)       (None, 224, 224, 64)      36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, 112, 112, 64)      0         
                                                                 
 block2_conv1 (Conv2D)       (None, 112, 112, 128)     73856     
                                                                 
 block2_conv2 (Conv2D)       (None, 112, 112, 128)     147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, 56, 56, 128)       0         
                                                                 
 block3_conv1 (Conv2D)       (None, 56, 56, 256)       295168    
                                                                 
 block3_conv2 (Conv2D)       (None, 56, 56, 256)       590080    
                                                                 
 block3_conv3 (Conv2D)       (None, 56, 56, 256)       590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, 28, 28, 256)       0         
                                                                 
 block4_conv1 (Conv2D)       (None, 28, 28, 512)       1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, 28, 28, 512)       2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, 28, 28, 512)       2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, 14, 14, 512)       0         
                                                                 
 block5_conv1 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, 14, 14, 512)       2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, 7, 7, 512)         0         
                                                                 
 flatten (Flatten)           (None, 25088)             0         
                                                                 
 fc1 (Dense)                 (None, 4096)              102764544 
                                                                 
 fc2 (Dense)                 (None, 4096)              16781312  
                                                                 
 dense_3 (Dense)             (None, 2)                 8194      
                                                                 
=================================================================
Total params: 134,268,738
Trainable params: 8,194
Non-trainable params: 134,260,544
_________________________________________________________________

Training the fine tuned VGG16 model

# compiling the model withe the optimizer, loss and metrics

model.compile(optimizer=Adam(learning_rate=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])

# fitting the model with the training and validation batches
# because of the pre trained network, we get an amazing accuracy

model.fit(x= train_batches, validation_data= valid_batches, epochs = 5, verbose= 2)

Epoch 1/5
100/100 - 400s - loss: 0.3298 - accuracy: 0.8680 - val_loss: 0.0962 - val_accuracy: 0.9550 - 400s/epoch - 4s/step
Epoch 2/5
100/100 - 361s - loss: 0.1089 - accuracy: 0.9630 - val_loss: 0.0561 - val_accuracy: 0.9900 - 361s/epoch - 4s/step
Epoch 3/5
100/100 - 362s - loss: 0.0726 - accuracy: 0.9810 - val_loss: 0.0481 - val_accuracy: 0.9750 - 362s/epoch - 4s/step
Epoch 4/5
100/100 - 361s - loss: 0.0521 - accuracy: 0.9860 - val_loss: 0.0389 - val_accuracy: 0.9950 - 361s/epoch - 4s/step
Epoch 5/5
100/100 - 376s - loss: 0.0389 - accuracy: 0.9870 - val_loss: 0.0382 - val_accuracy: 0.9900 - 376s/epoch - 4s/step

<keras.callbacks.History at 0x1fe8fc819a0>

Prediction / Inference

#getting the test_batch for prediction

predictions = model.predict(x= test_batches, verbose= 0)

# getting the classes of the test set
# note that we are not shuffling the test set so that we can compare with the predictions classes

test_batches.classes

array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])

# calling the confusion matrix function

cm = confusion_matrix(y_true = test_batches.classes, y_pred = np.argmax(predictions, axis = -1))

# checking which index is 0 and 1

test_batches.class_indices

{'cat': 0, 'dog': 1}

# plotting the confusion matrix

cm_plot_labels = ['cat', 'dog']

disp = ConfusionMatrixDisplay(confusion_matrix= cm, display_labels= cm_plot_labels)

disp.plot(cmap=plt.cm.Blues)
plt.show()

print(49/50)

0.98

We can see that there is a very high accuracy and the model correctly predicted both the classes with an accuracy of 98%

Training custom untrained data by any pre-trained model

MobileNet

• Light weight in size
• Great for mobile devices
• Size of VGG16 is 553MB where as MobileNet size is only 17MB
• MobileNet has 4.2 Million parameters
• MobileNet is not as accurate as VGG16, however only a small accuracy is the difference in tradeoff

# importing libraries

import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import categorical_crossentropy
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing import image
from tensorflow.keras.models import Model
from tensorflow.keras.applications import imagenet_utils
from sklearn.metrics import confusion_matrix
import itertools
import os
import shutil
import random
import matplotlib.pyplot as plt
%matplotlib inline

# checking for GPU

physical_devices = tf.config.experimental.list_physical_devices('GPU')
print("Num GPUs Available: ", len(physical_devices))
#tf.config.experimental.set_memory_growth(physical_devices[0], True)

Num GPUs Available:  0

# downloading the mobileNet pre-trained model
# mobilenet was trained on imagenet libraries

mobile = tf.keras.applications.mobilenet.MobileNet()

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet/mobilenet_1_0_224_tf.h5
17227776/17225924 [==============================] - 7s 0us/step
17235968/17225924 [==============================] - 7s 0us/step

# function to process the input image

img_path = r'C:\Users\tejas\Desktop\work\Portfolio_projects\tensorflow\data\MobileNet-samples'

def prepare_image(file):
    
    img = image.load_img(img_path + file, target_size=(224, 224)) # loading the image throug keras API load_img
    img_array = image.img_to_array(img) # converting the image to an array
    img_array_expanded_dims = np.expand_dims(img_array, axis=0) # expanding the dimension of the image array as expected by MobileNet
    return tf.keras.applications.mobilenet.preprocess_input(img_array_expanded_dims) # preprocessing the image to reduce the scale between -1 and 1

# passing a random image to check

from IPython.display import Image

Image(filename= img_path + '\\1.PNG', width=300,height=200) 

os.chdir("../data")
print(os.getcwd())

C:\Users\tejas\Desktop\work\Portfolio_projects\tensorflow\data

# passing the same image to the above preprocessing function 

# preprocessing the image
preprocessed_image = prepare_image('\\1.PNG')

# predicting the preprocessed image with mobilenet model
predictions = mobile.predict(preprocessed_image)

# calling decode_predictions keras API function to give the top 5 predictions out of the 1000 imagenet classes
results = imagenet_utils.decode_predictions(predictions)
results

Downloading data from https://storage.googleapis.com/download.tensorflow.org/data/imagenet_class_index.json
40960/35363 [==================================] - 0s 1us/step
49152/35363 [=========================================] - 0s 1us/step

[[('n01682714', 'American_chameleon', 0.57064366),
  ('n01694178', 'African_chameleon', 0.19318135),
  ('n01677366', 'common_iguana', 0.10827039),
  ('n01693334', 'green_lizard', 0.077062614),
  ('n01687978', 'agama', 0.023457762)]]

# moving on to a second image

Image(filename=img_path + '\\2.PNG', width=300,height=200)

# passing the same image to the above preprocessing function 
# using all the preprocessing steps same as above image

preprocessed_image = prepare_image('\\2.PNG')
predictions = mobile.predict(preprocessed_image)
results = imagenet_utils.decode_predictions(predictions)
results

[[('n07920052', 'espresso', 0.98888385),
  ('n07930864', 'cup', 0.00958599),
  ('n07932039', 'eggnog', 0.0007359698),
  ('n03063599', 'coffee_mug', 0.0006169288),
  ('n03063689', 'coffeepot', 3.1316125e-05)]]

Custom dataset (Unseen by imageNet library)

os.getcwd()

'C:\\Users\\tejas\\Desktop\\work\\Portfolio_projects\\tensorflow\\data'

# cloning the dataset from github source

!git clone https://github.com/ardamavi/Sign-Language-Digits-Dataset.git

Cloning into 'Sign-Language-Digits-Dataset'...
Updating files:   8% (170/2074)
Updating files:   9% (187/2074)
Updating files:  10% (208/2074)
Updating files:  11% (229/2074)
Updating files:  12% (249/2074)
Updating files:  12% (250/2074)
Updating files:  13% (270/2074)
Updating files:  14% (291/2074)
Updating files:  15% (312/2074)
Updating files:  16% (332/2074)
Updating files:  17% (353/2074)
Updating files:  18% (374/2074)
Updating files:  18% (392/2074)
Updating files:  19% (395/2074)
Updating files:  20% (415/2074)
Updating files:  21% (436/2074)
Updating files:  22% (457/2074)
Updating files:  23% (478/2074)
Updating files:  24% (498/2074)
Updating files:  24% (512/2074)
Updating files:  25% (519/2074)
Updating files:  26% (540/2074)
Updating files:  27% (560/2074)
Updating files:  27% (562/2074)
Updating files:  28% (581/2074)
Updating files:  29% (602/2074)
Updating files:  30% (623/2074)
Updating files:  31% (643/2074)
Updating files:  32% (664/2074)
Updating files:  32% (677/2074)
Updating files:  33% (685/2074)
Updating files:  34% (706/2074)
Updating files:  35% (726/2074)
Updating files:  36% (747/2074)
Updating files:  37% (768/2074)
Updating files:  38% (789/2074)
Updating files:  39% (809/2074)
Updating files:  39% (829/2074)
Updating files:  40% (830/2074)
Updating files:  41% (851/2074)
Updating files:  42% (872/2074)
Updating files:  43% (892/2074)
Updating files:  44% (913/2074)
Updating files:  45% (934/2074)
Updating files:  46% (955/2074)
Updating files:  47% (975/2074)
Updating files:  47% (995/2074)
Updating files:  48% (996/2074)
Updating files:  49% (1017/2074)
Updating files:  50% (1037/2074)
Updating files:  51% (1058/2074)
Updating files:  52% (1079/2074)
Updating files:  52% (1082/2074)
Updating files:  52% (1088/2074)
Updating files:  52% (1097/2074)
Updating files:  53% (1100/2074)
Updating files:  54% (1120/2074)
Updating files:  55% (1141/2074)
Updating files:  55% (1157/2074)
Updating files:  56% (1162/2074)
Updating files:  57% (1183/2074)
Updating files:  58% (1203/2074)
Updating files:  59% (1224/2074)
Updating files:  60% (1245/2074)
Updating files:  60% (1255/2074)
Updating files:  61% (1266/2074)
Updating files:  62% (1286/2074)
Updating files:  63% (1307/2074)
Updating files:  64% (1328/2074)
Updating files:  65% (1349/2074)
Updating files:  65% (1356/2074)
Updating files:  66% (1369/2074)
Updating files:  67% (1390/2074)
Updating files:  68% (1411/2074)
Updating files:  69% (1432/2074)
Updating files:  70% (1452/2074)
Updating files:  71% (1473/2074)
Updating files:  72% (1494/2074)
Updating files:  73% (1515/2074)
Updating files:  73% (1523/2074)
Updating files:  74% (1535/2074)
Updating files:  75% (1556/2074)
Updating files:  76% (1577/2074)
Updating files:  77% (1597/2074)
Updating files:  78% (1618/2074)
Updating files:  79% (1639/2074)
Updating files:  80% (1660/2074)
Updating files:  80% (1672/2074)
Updating files:  80% (1677/2074)
Updating files:  81% (1680/2074)
Updating files:  82% (1701/2074)
Updating files:  83% (1722/2074)
Updating files:  83% (1734/2074)
Updating files:  84% (1743/2074)
Updating files:  85% (1763/2074)
Updating files:  85% (1765/2074)
Updating files:  86% (1784/2074)
Updating files:  86% (1794/2074)
Updating files:  87% (1805/2074)
Updating files:  88% (1826/2074)
Updating files:  89% (1846/2074)
Updating files:  90% (1867/2074)
Updating files:  90% (1872/2074)
Updating files:  91% (1888/2074)
Updating files:  91% (1894/2074)
Updating files:  92% (1909/2074)
Updating files:  93% (1929/2074)
Updating files:  93% (1930/2074)
Updating files:  93% (1937/2074)
Updating files:  94% (1950/2074)
Updating files:  95% (1971/2074)
Updating files:  96% (1992/2074)
Updating files:  97% (2012/2074)
Updating files:  98% (2033/2074)
Updating files:  99% (2054/2074)
Updating files: 100% (2074/2074)
Updating files: 100% (2074/2074), done.

# organze the data into train, test and valid dirs

# changing the directory to Sign-Language-Digits-Dataset
# checking if the directory is already present on disk, if not it will create new directories
os.chdir('Sign-Language-Digits-Dataset')
if os.path.isdir('train/0/') is False: 
    os.mkdir('train')
    os.mkdir('valid')
    os.mkdir('test')

    # looping through the 10 classes and making directories for the ten classes in train, valid and test directories
    # moving all the images to the train directories 
    for i in range(0, 10):
        shutil.move(f'{i}', 'train')
        os.mkdir(f'valid/{i}')
        os.mkdir(f'test/{i}')

        # taking 30 random samples from the train dir and moving them to the valid directory
        valid_samples = random.sample(os.listdir(f'train/{i}'), 30)
        for j in valid_samples:
            shutil.move(f'train/{i}/{j}', f'valid/{i}')

        # taking 5 random samples from the train dir and moving them to the valid directory
        test_samples = random.sample(os.listdir(f'train/{i}'), 5)
        for k in test_samples:
            shutil.move(f'train/{i}/{k}', f'test/{i}')
            
# getting back to the current directory
os.chdir('../..')

# assinging the data directories

train_path = 'data/Sign-Language-Digits-Dataset/train'
valid_path = 'data/Sign-Language-Digits-Dataset/valid'
test_path = 'data/Sign-Language-Digits-Dataset/test'

# using mobilenet preprocessor and creating a generator function to pass the images one by one to the model

train_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(
    directory=train_path, target_size=(224,224), batch_size=10)

valid_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(
    directory=valid_path, target_size=(224,224), batch_size=10)

test_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(
    directory=test_path, target_size=(224,224), batch_size=10, shuffle=False) # test set shuffle=False

Found 1712 images belonging to 10 classes.
Found 300 images belonging to 10 classes.
Found 50 images belonging to 10 classes.

Modify the Model

#downloading the model from keras applications

mobile = tf.keras.applications.mobilenet.MobileNet()

mobile.summary()

Model: "mobilenet_1.00_224"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 conv1 (Conv2D)              (None, 112, 112, 32)      864       
                                                                 
 conv1_bn (BatchNormalizatio  (None, 112, 112, 32)     128       
 n)                                                              
                                                                 
 conv1_relu (ReLU)           (None, 112, 112, 32)      0         
                                                                 
 conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)     288       
                                                                 
 conv_dw_1_bn (BatchNormaliz  (None, 112, 112, 32)     128       
 ation)                                                          
                                                                 
 conv_dw_1_relu (ReLU)       (None, 112, 112, 32)      0         
                                                                 
 conv_pw_1 (Conv2D)          (None, 112, 112, 64)      2048      
                                                                 
 conv_pw_1_bn (BatchNormaliz  (None, 112, 112, 64)     256       
 ation)                                                          
                                                                 
 conv_pw_1_relu (ReLU)       (None, 112, 112, 64)      0         
                                                                 
 conv_pad_2 (ZeroPadding2D)  (None, 113, 113, 64)      0         
                                                                 
 conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)       576       
                                                                 
 conv_dw_2_bn (BatchNormaliz  (None, 56, 56, 64)       256       
 ation)                                                          
                                                                 
 conv_dw_2_relu (ReLU)       (None, 56, 56, 64)        0         
                                                                 
 conv_pw_2 (Conv2D)          (None, 56, 56, 128)       8192      
                                                                 
 conv_pw_2_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_2_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)      1152      
                                                                 
 conv_dw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pw_3 (Conv2D)          (None, 56, 56, 128)       16384     
                                                                 
 conv_pw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pad_4 (ZeroPadding2D)  (None, 57, 57, 128)       0         
                                                                 
 conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)      1152      
                                                                 
 conv_dw_4_bn (BatchNormaliz  (None, 28, 28, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_4_relu (ReLU)       (None, 28, 28, 128)       0         
                                                                 
 conv_pw_4 (Conv2D)          (None, 28, 28, 256)       32768     
                                                                 
 conv_pw_4_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_4_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)      2304      
                                                                 
 conv_dw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pw_5 (Conv2D)          (None, 28, 28, 256)       65536     
                                                                 
 conv_pw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pad_6 (ZeroPadding2D)  (None, 29, 29, 256)       0         
                                                                 
 conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)      2304      
                                                                 
 conv_dw_6_bn (BatchNormaliz  (None, 14, 14, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_6_relu (ReLU)       (None, 14, 14, 256)       0         
                                                                 
 conv_pw_6 (Conv2D)          (None, 14, 14, 512)       131072    
                                                                 
 conv_pw_6_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_6_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_7 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_8 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_9 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_10 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_10 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_dw_11 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_11 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)      0         
                                                                 
 conv_dw_12 (DepthwiseConv2D  (None, 7, 7, 512)        4608      
 )                                                               
                                                                 
 conv_dw_12_bn (BatchNormali  (None, 7, 7, 512)        2048      
 zation)                                                         
                                                                 
 conv_dw_12_relu (ReLU)      (None, 7, 7, 512)         0         
                                                                 
 conv_pw_12 (Conv2D)         (None, 7, 7, 1024)        524288    
                                                                 
 conv_pw_12_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_12_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_dw_13 (DepthwiseConv2D  (None, 7, 7, 1024)       9216      
 )                                                               
                                                                 
 conv_dw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_dw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_pw_13 (Conv2D)         (None, 7, 7, 1024)        1048576   
                                                                 
 conv_pw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 global_average_pooling2d_3   (None, 1, 1, 1024)       0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout (Dropout)           (None, 1, 1, 1024)        0         
                                                                 
 conv_preds (Conv2D)         (None, 1, 1, 1000)        1025000   
                                                                 
 reshape_2 (Reshape)         (None, 1000)              0         
                                                                 
 predictions (Activation)    (None, 1000)              0         
                                                                 
=================================================================
Total params: 4,253,864
Trainable params: 4,231,976
Non-trainable params: 21,888
_________________________________________________________________

# keeping all the layers except the last 6 layers
# this is done by experimentation to check what works fine with the model
# we found that removing the last 6 layers in the network gives a better accuracy
# The last dense layer should have the total classes we have for prediction, hence removing the 1000 prediction classes
# And adding a customised 10 unit prediction output

# this is basically the input layer
x = mobile.layers[-2].output

# setting our output layer
# The mobilenet is a functional model and we will be using functional model instead of converting it to a sequential model

# creating an output layer with 10 units as we have 10 classes
output = Dense(units=10, activation='softmax')(x)

# creating a model with the functional API

model = Model(inputs = mobile.input, outputs = output)

# by experimentation we can find if we freeze all the layers except the last 23 layers we can get good results
# so we will be freezing all the layers except the last 23 layers
# so out of the total 88 layers in mobilenet the last 23 layers will be trainable, rest will be freezed

for layer in model.layers[:-23]:
    layer.trainable = False

model.summary()

Model: "model_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, 224, 224, 3)]     0         
                                                                 
 conv1 (Conv2D)              (None, 112, 112, 32)      864       
                                                                 
 conv1_bn (BatchNormalizatio  (None, 112, 112, 32)     128       
 n)                                                              
                                                                 
 conv1_relu (ReLU)           (None, 112, 112, 32)      0         
                                                                 
 conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)     288       
                                                                 
 conv_dw_1_bn (BatchNormaliz  (None, 112, 112, 32)     128       
 ation)                                                          
                                                                 
 conv_dw_1_relu (ReLU)       (None, 112, 112, 32)      0         
                                                                 
 conv_pw_1 (Conv2D)          (None, 112, 112, 64)      2048      
                                                                 
 conv_pw_1_bn (BatchNormaliz  (None, 112, 112, 64)     256       
 ation)                                                          
                                                                 
 conv_pw_1_relu (ReLU)       (None, 112, 112, 64)      0         
                                                                 
 conv_pad_2 (ZeroPadding2D)  (None, 113, 113, 64)      0         
                                                                 
 conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)       576       
                                                                 
 conv_dw_2_bn (BatchNormaliz  (None, 56, 56, 64)       256       
 ation)                                                          
                                                                 
 conv_dw_2_relu (ReLU)       (None, 56, 56, 64)        0         
                                                                 
 conv_pw_2 (Conv2D)          (None, 56, 56, 128)       8192      
                                                                 
 conv_pw_2_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_2_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)      1152      
                                                                 
 conv_dw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pw_3 (Conv2D)          (None, 56, 56, 128)       16384     
                                                                 
 conv_pw_3_bn (BatchNormaliz  (None, 56, 56, 128)      512       
 ation)                                                          
                                                                 
 conv_pw_3_relu (ReLU)       (None, 56, 56, 128)       0         
                                                                 
 conv_pad_4 (ZeroPadding2D)  (None, 57, 57, 128)       0         
                                                                 
 conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)      1152      
                                                                 
 conv_dw_4_bn (BatchNormaliz  (None, 28, 28, 128)      512       
 ation)                                                          
                                                                 
 conv_dw_4_relu (ReLU)       (None, 28, 28, 128)       0         
                                                                 
 conv_pw_4 (Conv2D)          (None, 28, 28, 256)       32768     
                                                                 
 conv_pw_4_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_4_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)      2304      
                                                                 
 conv_dw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pw_5 (Conv2D)          (None, 28, 28, 256)       65536     
                                                                 
 conv_pw_5_bn (BatchNormaliz  (None, 28, 28, 256)      1024      
 ation)                                                          
                                                                 
 conv_pw_5_relu (ReLU)       (None, 28, 28, 256)       0         
                                                                 
 conv_pad_6 (ZeroPadding2D)  (None, 29, 29, 256)       0         
                                                                 
 conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)      2304      
                                                                 
 conv_dw_6_bn (BatchNormaliz  (None, 14, 14, 256)      1024      
 ation)                                                          
                                                                 
 conv_dw_6_relu (ReLU)       (None, 14, 14, 256)       0         
                                                                 
 conv_pw_6 (Conv2D)          (None, 14, 14, 512)       131072    
                                                                 
 conv_pw_6_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_6_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_7 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_7_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_7_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_8 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_8_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_8_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)      4608      
                                                                 
 conv_dw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_dw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_pw_9 (Conv2D)          (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_9_bn (BatchNormaliz  (None, 14, 14, 512)      2048      
 ation)                                                          
                                                                 
 conv_pw_9_relu (ReLU)       (None, 14, 14, 512)       0         
                                                                 
 conv_dw_10 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_10 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_10_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_10_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_dw_11 (DepthwiseConv2D  (None, 14, 14, 512)      4608      
 )                                                               
                                                                 
 conv_dw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_dw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pw_11 (Conv2D)         (None, 14, 14, 512)       262144    
                                                                 
 conv_pw_11_bn (BatchNormali  (None, 14, 14, 512)      2048      
 zation)                                                         
                                                                 
 conv_pw_11_relu (ReLU)      (None, 14, 14, 512)       0         
                                                                 
 conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)      0         
                                                                 
 conv_dw_12 (DepthwiseConv2D  (None, 7, 7, 512)        4608      
 )                                                               
                                                                 
 conv_dw_12_bn (BatchNormali  (None, 7, 7, 512)        2048      
 zation)                                                         
                                                                 
 conv_dw_12_relu (ReLU)      (None, 7, 7, 512)         0         
                                                                 
 conv_pw_12 (Conv2D)         (None, 7, 7, 1024)        524288    
                                                                 
 conv_pw_12_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_12_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_dw_13 (DepthwiseConv2D  (None, 7, 7, 1024)       9216      
 )                                                               
                                                                 
 conv_dw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_dw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 conv_pw_13 (Conv2D)         (None, 7, 7, 1024)        1048576   
                                                                 
 conv_pw_13_bn (BatchNormali  (None, 7, 7, 1024)       4096      
 zation)                                                         
                                                                 
 conv_pw_13_relu (ReLU)      (None, 7, 7, 1024)        0         
                                                                 
 global_average_pooling2d_3   (None, 1, 1, 1024)       0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout (Dropout)           (None, 1, 1, 1024)        0         
                                                                 
 conv_preds (Conv2D)         (None, 1, 1, 1000)        1025000   
                                                                 
 reshape_2 (Reshape)         (None, 1000)              0         
                                                                 
 dense_7 (Dense)             (None, 10)                10010     
                                                                 
=================================================================
Total params: 4,263,874
Trainable params: 2,893,058
Non-trainable params: 1,370,816
_________________________________________________________________

Training the model

# compiling the model

model.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])

# fitting and training the model
# should run for more than 30 epochs to see good results

model.fit(x= train_batches, validation_data= valid_batches, epochs= 10, verbose= 2)

Epoch 1/10
172/172 - 121s - loss: 0.8125 - accuracy: 0.7710 - val_loss: 2.0060 - val_accuracy: 0.5967 - 121s/epoch - 706ms/step
Epoch 2/10
172/172 - 112s - loss: 0.1002 - accuracy: 0.9702 - val_loss: 0.5255 - val_accuracy: 0.8700 - 112s/epoch - 651ms/step
Epoch 3/10
172/172 - 108s - loss: 0.0594 - accuracy: 0.9807 - val_loss: 0.0998 - val_accuracy: 0.9700 - 108s/epoch - 626ms/step
Epoch 4/10
172/172 - 110s - loss: 0.0349 - accuracy: 0.9866 - val_loss: 0.1292 - val_accuracy: 0.9567 - 110s/epoch - 640ms/step
Epoch 5/10
172/172 - 105s - loss: 0.0281 - accuracy: 0.9889 - val_loss: 0.1004 - val_accuracy: 0.9667 - 105s/epoch - 611ms/step
Epoch 6/10
172/172 - 106s - loss: 0.0255 - accuracy: 0.9930 - val_loss: 0.3123 - val_accuracy: 0.9133 - 106s/epoch - 617ms/step
Epoch 7/10
172/172 - 101s - loss: 0.0250 - accuracy: 0.9918 - val_loss: 0.0983 - val_accuracy: 0.9733 - 101s/epoch - 589ms/step
Epoch 8/10
172/172 - 105s - loss: 0.0300 - accuracy: 0.9901 - val_loss: 0.1496 - val_accuracy: 0.9567 - 105s/epoch - 613ms/step
Epoch 9/10
172/172 - 106s - loss: 0.0588 - accuracy: 0.9836 - val_loss: 0.1464 - val_accuracy: 0.9533 - 106s/epoch - 613ms/step
Epoch 10/10
172/172 - 114s - loss: 0.0252 - accuracy: 0.9912 - val_loss: 0.1744 - val_accuracy: 0.9500 - 114s/epoch - 664ms/step

<keras.callbacks.History at 0x1fe90973340>

Predict Sign Language

# getting the labels of the test set

test_labels = test_batches.classes
test_labels

array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4,
       4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8,
       8, 9, 9, 9, 9, 9])

# using model.predict to predict the output

predictions = model.predict(x=test_batches, steps=len(test_batches), verbose=0)

# confusion matrix

cm = confusion_matrix(y_true=test_labels, y_pred=predictions.argmax(axis=1))

# getting the indices of the test_batches

test_batches.class_indices

{'0': 0,
 '1': 1,
 '2': 2,
 '3': 3,
 '4': 4,
 '5': 5,
 '6': 6,
 '7': 7,
 '8': 8,
 '9': 9}

# plotting the confusion matrix

cm_plot_labels = ['0','1','2','3','4','5','6','7','8','9']

disp = ConfusionMatrixDisplay(confusion_matrix= cm, display_labels= cm_plot_labels)

disp.plot(cmap=plt.cm.Blues)
plt.show()

we get an amazing prediction for the test set with high accuracy

Data Agumentation

# importing libraries

import matplotlib.pyplot as plt
import numpy as np
import os
import random
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
%matplotlib inline

# function to plot the image

def plotImages(images_arr):
    fig, axes = plt.subplots(1, 10, figsize=(20,20))
    axes = axes.flatten()
    for img, ax in zip( images_arr, axes):
        ax.imshow(img)
        ax.axis('off')
    plt.tight_layout()
    plt.show()

# Using ImageDataGenerator to Augument the images as a generator function
# View the documentation to set values for the parameters

gen = ImageDataGenerator(rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.15, zoom_range=0.1, 
    channel_shift_range=10., horizontal_flip=True)

# We choose a random dog image from the directory

chosen_image = random.choice(os.listdir('data/dogs-vs-cats/train/dog'))

# getting the path of the image
image_path = 'data/dogs-vs-cats/train/dog/' + chosen_image

# reading the dimensions of the image and reading the image
image = np.expand_dims(plt.imread(image_path),0)

# plotting the image
plt.imshow(image[0])

<matplotlib.image.AxesImage at 0x1fea1baf790>

# generate batches of augmented images from the original image.

aug_iter = gen.flow(image)

# get ten samples of the augmented images as a iterator function.

aug_images = [next(aug_iter)[0].astype(np.uint8) for i in range(10)]

# plotting the agumented images

plotImages(aug_images)

# fit_generator function allows for data augmentation and data generators.

gen.fit(image)

save_here = r'C:\Users\tejas\Desktop\work\Portfolio_projects\tensorflow\data\dogs-vs-cats\train\dog'

#saving the augmented images to the respective directories

for x, val in zip(gen.flow(image,  #image we chose
        save_to_dir= save_here,     #this is where we figure out where to save
         save_prefix= 'dog',        # it will save the images as 'dog_0912' some number for every new augmented image
        save_format='jpeg'),range(10)) :     # here we define a range because we want 10 augmented images otherwise it will keep looping forever I think
    pass